<!DOCTYPE html>
<html lang="zh">
    <head>
        <meta charset="utf-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
        <title>信息论学习笔记 | ChenDong Zhu&#39;s Blog</title><meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="robots" content="noodp" />
<meta name="Description" content="学习信息论的笔记"><link rel="prev" href="/2021/09/%E5%8C%BA%E5%9D%97%E9%93%BE%E4%B8%8E%E5%8C%BA%E5%9D%97%E9%93%BE%E5%AE%89%E5%85%A8/" /><link rel="next" href="/2021/09/algebra/" /><link rel="canonical" href="/2021/09/%E4%BF%A1%E6%81%AF%E8%AE%BA/" />
<link rel="shortcut icon" type="image/x-icon" href="/favicon.ico" />
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
<link rel="manifest" href="/site.webmanifest">
<link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5">
<meta name="msapplication-TileColor" content="#da532c">
<meta name="theme-color" content="#ffffff"><meta property="og:title" content="信息论学习笔记" />
<meta property="og:description" content="学习信息论的笔记" />
<meta property="og:type" content="article" />
<meta property="og:url" content="/2021/09/%E4%BF%A1%E6%81%AF%E8%AE%BA/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2021-09-15T09:00:00+08:00" />
<meta property="article:modified_time" content="2021-09-15T09:00:00+08:00" />

<meta name="twitter:card" content="summary"/>
<meta name="twitter:title" content="信息论学习笔记"/>
<meta name="twitter:description" content="学习信息论的笔记"/>
<script type="application/ld+json">
    {
        "@context": "http://schema.org",
        "@type": "BlogPosting",
        "headline": "信息论学习笔记",
        "mainEntityOfPage": {
            "@type": "WebPage",
            "@id": "\/2021\/09\/%E4%BF%A1%E6%81%AF%E8%AE%BA\/"
        },"image": {
                "@type": "ImageObject",
                "url": "\/cover.png",
                "width":  800 ,
                "height":  600 
            },"genre": "posts","keywords": "信息论","wordcount":  1319 ,
        "url": "\/2021\/09\/%E4%BF%A1%E6%81%AF%E8%AE%BA\/","datePublished": "2021-09-15T09:00:00\u002b08:00","dateModified": "2021-09-15T09:00:00\u002b08:00","license": "This work is licensed under a Creative Commons Attribution-NonCommercial 4.0 International License.","publisher": {
                "@type": "Organization",
                "name": "xxxx",
                "logo": {
                "@type": "ImageObject",
                "url": "\/logo.png",
                "width":  127 ,
                "height":  40 
                }
            },"description": "学习信息论的笔记"
    }
    </script><link rel="stylesheet" href="/css/style.min.css"><link rel="stylesheet" href="/css/lib/fontawesome-free/all.min.css"><link rel="stylesheet" href="/css/lib/animate/animate.min.css"></head>
    <body><script>
            window.isDark = (window.localStorage && window.localStorage.getItem('theme')) === 'dark';
            window.isDark && document.body.classList.add('dark-theme');
        </script><div class="wrapper"><nav class="navbar">
    <div class="navbar-container">
        <div class="navbar-header animated bounceIn">
            <a href="/">ChenDong Zhu&#39;s Blog</a>
        </div>
        <div class="navbar-menu"><a class="menu-item" href="/posts" title="">文章</a><a class="menu-item" href="/tags" title="">标签</a><a class="menu-item" href="/categories" title="">分类</a><a class="menu-item" href="/about" title="">关于</a><a class="menu-item" href="https://hugo-loveit-en.netlify.com" title="English"><i class="fas fa-language fa-fw"></i></a><a href="javascript:void(0);" class="theme-switch"><i class="fas fa-adjust fa-rotate-180 fa-fw" title="切换主题"></i></a>
        </div>
    </div>
</nav><nav class="navbar-mobile">
    <div class="navbar-container">
        <div class="navbar-header">
            <div class="navbar-header-title animated bounceIn">
                <a href="/">ChenDong Zhu&#39;s Blog</a>
            </div>
            <div class="menu-toggle" id="menu-toggle">
                <span></span><span></span><span></span>
            </div>
        </div>
        <div class="navbar-menu" id="mobile-menu"><a class="menu-item" href="/posts" title="">文章</a><a class="menu-item" href="/tags" title="">标签</a><a class="menu-item" href="/categories" title="">分类</a><a class="menu-item" href="/about" title="">关于</a><a class="menu-item" href="https://hugo-loveit-en.netlify.com" title="English"></a><a href="javascript:void(0);" class="theme-switch"><i class="fas fa-adjust fa-rotate-180 fa-fw" title="切换主题"></i></a>
        </div>
    </div>
</nav><main class="main">
                <div class="container"><article class="page"><h1 class="post-title animated flipInX">信息论学习笔记</h1><div class="post-meta">
            <div class="post-meta-main"><a class="author" href="/" rel="author" target="_blank">
                    <i class="fas fa-user-circle fa-fw"></i>ChenDong Zhu
                </a>&nbsp;<span class="post-category">收录于&nbsp;<i class="far fa-folder fa-fw"></i><a href="/categories/%E5%AD%A6%E4%B9%A0/">学习</a>&nbsp;</span></div>
            <div class="post-meta-other"><i class="far fa-calendar-alt fa-fw"></i><time datetime=2021-09-15>2021-09-15</time>&nbsp;
                <i class="fas fa-pencil-alt fa-fw"></i>约 1319 字&nbsp;
                <i class="far fa-clock fa-fw"></i>预计阅读 3 分钟&nbsp;</div>
        </div><div class="post-toc" id="post-toc">
                <h2 class="post-toc-title">目录</h2>
                <div class="post-toc-content"><nav id="TableOfContents">
  <ul>
    <li><a href="#背景">背景</a></li>
    <li><a href="#lessen-1-overview">Lessen 1 Overview</a>
      <ul>
        <li><a href="#shannon的三个理论">Shannon的三个理论</a></li>
        <li><a href="#推荐阅读书目">推荐阅读书目</a></li>
        <li><a href="#什么是信息">什么是信息</a></li>
        <li><a href="#信息的表示">信息的表示</a></li>
        <li><a href="#字典和码表codebook">字典和码表（Codebook）</a></li>
        <li><a href="#compactness-of-code码表的压缩">Compactness of code码表的压缩</a>
          <ul>
            <li><a href="#code1">Code1</a></li>
            <li><a href="#code2">Code2</a></li>
          </ul>
        </li>
        <li><a href="#font-colorredhow-to-measure-informationfont"><font color="red">How to measure information</font></a></li>
        <li><a href="#shannon的贡献">Shannon的贡献</a></li>
        <li><a href="#how-to-measure-information-2">How to measure information 2</a></li>
        <li><a href="#data-transmission-over-noisy-channel-在有噪声下数据的传输">Data transmission over noisy channel 在有噪声下数据的传输</a></li>
      </ul>
    </li>
  </ul>
</nav></div>
            </div>
            <div class="post-toc-mobile" id="post-toc-mobile">
                <details>
                    <summary>
                        <div class="post-toc-title">
                            <span>目录</span>
                            <span><i class="details icon fas fa-angle-down"></i></span>
                        </div>
                    </summary>
                    <div class="post-toc-content"><nav id="TableOfContentsMobile">
  <ul>
    <li><a href="#背景">背景</a></li>
    <li><a href="#lessen-1-overview">Lessen 1 Overview</a>
      <ul>
        <li><a href="#shannon的三个理论">Shannon的三个理论</a></li>
        <li><a href="#推荐阅读书目">推荐阅读书目</a></li>
        <li><a href="#什么是信息">什么是信息</a></li>
        <li><a href="#信息的表示">信息的表示</a></li>
        <li><a href="#字典和码表codebook">字典和码表（Codebook）</a></li>
        <li><a href="#compactness-of-code码表的压缩">Compactness of code码表的压缩</a>
          <ul>
            <li><a href="#code1">Code1</a></li>
            <li><a href="#code2">Code2</a></li>
          </ul>
        </li>
        <li><a href="#font-colorredhow-to-measure-informationfont"><font color="red">How to measure information</font></a></li>
        <li><a href="#shannon的贡献">Shannon的贡献</a></li>
        <li><a href="#how-to-measure-information-2">How to measure information 2</a></li>
        <li><a href="#data-transmission-over-noisy-channel-在有噪声下数据的传输">Data transmission over noisy channel 在有噪声下数据的传输</a></li>
      </ul>
    </li>
  </ul>
</nav></div>
                </details>
            </div><div class="post-content"><a class="post-dummy-target" id="背景"></a><h2>背景</h2>
<p>使用教材：<a href="https://www.jianguoyun.com/p/Db8AVqQQnd7rCRjc-Y4E" target="_blank">An introduction to single user information theory</a></p>
<p>课程源网址：<a href="http://ocw.nctu.edu.tw/course_detail.php?bgid=8&amp;gid=0&amp;nid=612" target="_blank">台湾国立交通大学-消息理论（Information Theory）</a></p>
<p>b站视频网址：<a href="https://www.bilibili.com/video/BV14N41197bN?p=2" target="_blank">BV14N41197bN</a></p>
<p>教师：陈伯宁</p>
<a class="post-dummy-target" id="lessen-1-overview"></a><h2>Lessen 1 Overview</h2>
<a class="post-dummy-target" id="shannon的三个理论"></a><h3>Shannon的三个理论</h3>
<ul>
<li>Shannon&rsquo;s first theory (source coding theory，讲压缩的)</li>
<li>Shannon&rsquo;s second theory (channel，讲信道的)</li>
<li>Shannon&rsquo;s third theory (rate-distortion，讲传输速率和噪声的关系)</li>
</ul>
<p>Shannon用了random coding argument（<font color="red">概率期望</font>的方式）来看待信息，<font color="red">开辟了一种新的证明&amp;看待问题的方法</font></p>
<a class="post-dummy-target" id="推荐阅读书目"></a><h3>推荐阅读书目</h3>
<ul>
<li><em>A Student&rsquo;s Guide to Coding and Information Theory, Stefan M. Moser and Po-Ning  Chen</em></li>
<li><em>Elements of Infomation Theory</em></li>
<li><em>Information-Spectrum Method in Information Theory, Te Sun Han</em></li>
</ul>
<a class="post-dummy-target" id="什么是信息"></a><h3>什么是信息</h3>
<p>Shannon认为信息就是<font color="red">Uncertainty</font>（不确定性），Information is a message that is previously uncertainty to receivers.信息就是接收者不知道的消息。</p>
<a class="post-dummy-target" id="信息的表示"></a><h3>信息的表示</h3>
<ul>
<li>在人的生产生活中，用语言表示信息，如英语(ABCD)，如中文（你我他）</li>
<li>在计算机中用Code(010101110100)来表示信息</li>
</ul>
<p>即信息符号(Symbol)化</p>
<a class="post-dummy-target" id="字典和码表codebook"></a><h3>字典和码表（Codebook）</h3>
<p>即所有人都事先知道信息的编码方式，但接收者不知道下一个Symbol会接收到什么。</p>
<a class="post-dummy-target" id="compactness-of-code码表的压缩"></a><h3>Compactness of code码表的压缩</h3>
<blockquote>
<p>Question : 如何找到一个最佳的码表，在该种编码方式下平均码长最小？</p>
<p>Answer of Engineer: 都尝试一遍</p>
<p>Answer of Shannon(Alternative Approach) : 俺不知道，但俺知道平均码长最小是多少，这样我就能知道一个编码方式距离最优有多远</p>
</blockquote>
<p>（预先知道每个Symbol出现的概率，如假设一共4个Symbol,A出现概率是$\frac{1}{2}$，B出现概率是$\frac{1}{4}$，C出现概率是$\frac{1}{8}$,D出现概率是$\frac{1}{8}$）</p>
<a class="post-dummy-target" id="code1"></a><h4>Code1</h4>
<table>
<thead>
<tr>
<th>Symbol</th>
<th>Code</th>
</tr>
</thead>
<tbody>
<tr>
<td>A</td>
<td>00</td>
</tr>
<tr>
<td>B</td>
<td>01</td>
</tr>
<tr>
<td>C</td>
<td>10</td>
</tr>
<tr>
<td>D</td>
<td>11</td>
</tr>
</tbody>
</table>
<p>该方式平均码长是2</p>
<a class="post-dummy-target" id="code2"></a><h4>Code2</h4>
<table>
<thead>
<tr>
<th>Symbol</th>
<th>Code</th>
</tr>
</thead>
<tbody>
<tr>
<td>A</td>
<td>0</td>
</tr>
<tr>
<td>B</td>
<td>10</td>
</tr>
<tr>
<td>C</td>
<td>110</td>
</tr>
<tr>
<td>D</td>
<td>111</td>
</tr>
</tbody>
</table>
<p>该方式平均码长是
$$
average = 1*\frac{1}{2}+2*\frac{1}{4}+3*\frac{1}{8}+3*\frac{1}{8}=\frac{7}{4}
$$
明显Code2优于Code1</p>
<a class="post-dummy-target" id="font-colorredhow-to-measure-informationfont"></a><h3><font color="red">How to measure information</font></h3>
<p>为了求最小码长，Shannon提出了以下3个公理</p>
<ul>
<li>Monotonicity in event probability. 事件几率越大，所携带的信息量越小</li>
<li>Additivity. 两个时间独立时，信息量可以相加。</li>
<li>Continuity(连续性).信息量关于几率的函数时连续的</li>
</ul>
<p>然后更具以上特性，总结出，满足以上公理的唯一信息-几率函数为
$$
self-information\ of\ an\ event = log_2\frac{1}{event probabily} bits
$$
并且用<font color="red">entropy</font>（熵）这个概念表示信息量</p>
<a class="post-dummy-target" id="shannon的贡献"></a><h3>Shannon的贡献</h3>
<ul>
<li>奠定了信息论基础</li>
<li>Shannon创立信息论的思考方式（下定义——思考性质——从性质抽象出规律）以及其从期望的角度对问题的思考可以指引其他学科</li>
</ul>
<a class="post-dummy-target" id="how-to-measure-information-2"></a><h3>How to measure information 2</h3>
<ul>
<li>平均码长只是一种度量方式</li>
<li>对于某些应用，需要最长码长最小</li>
<li>码长与成本之前可能存在函数关系（有学者做这方面的研究）</li>
</ul>
<a class="post-dummy-target" id="data-transmission-over-noisy-channel-在有噪声下数据的传输"></a><h3>Data transmission over noisy channel 在有噪声下数据的传输</h3>
<ul>
<li>1 &mdash;-&gt; 111</li>
<li>0 &mdash;-&gt; 000</li>
</ul>
<p>(通过冗余传输来保证正确率)，以上传输3个重复的码one information bit per three channel bits</p>
<p><figure><img src="/svg/loading.min.svg" data-sizes="auto" data-src="/images/information-theory/figure1.png" alt="传输过程" class="lazyload"><figcaption class="image-caption">传输过程</figcaption></figure></p>
<p>在Source Encoder阶段，需要把原始Symbols尽可能压缩，但是在Channel Encoder阶段，需要add redundancy尽可能得新增冗余来保护通信系统对抗噪声使系统具有reliably</p>
<p>压缩率的定义为
$$
Compression\ rate = number\ of\ U&rsquo;s\ per\ number\ of\ Z&rsquo;s
$$
Shannon证明了:
$$
Min(Compression\ rate)=entropy\ of\ Z=\sum_{z \in Z}F_Z(z)log_{|u|}\frac{1}{P_Z(z)}\ code\ bit
$$
(注意大写Z和小写z不一样)</p>
<p>最好的压缩方法，<font color="red">最后出来的Symbol序列中每个Symbol概率应该都是相等的</font>,比如上面code1的方式，计算出来0和1的概率都是$\frac{1}{2}$</p>
</div><div class="post-footer" id="post-footer">
    <div class="post-info">
        <div class="post-info-line">
            <div class="post-info-mod">
                <span>本文于 2021-09-15 更新</span>
            </div>
            <div class="post-info-license"></div>
        </div>
        <div class="post-info-line">
            <div class="post-info-md"></div>
            <div class="post-info-share"><span><a href="//twitter.com/share?url=%2f2021%2f09%2f%25E4%25BF%25A1%25E6%2581%25AF%25E8%25AE%25BA%2f&amp;text=%e4%bf%a1%e6%81%af%e8%ae%ba%e5%ad%a6%e4%b9%a0%e7%ac%94%e8%ae%b0&amp;via=" target="_blank" title="分享到 Twitter">
            <i class="fab fa-twitter fa-fw"></i>
        </a><a href="//www.facebook.com/sharer/sharer.php?u=%2f2021%2f09%2f%25E4%25BF%25A1%25E6%2581%25AF%25E8%25AE%25BA%2f" target="_blank" title="分享到 Facebook">
            <i class="fab fa-facebook-square fa-fw"></i>
        </a><a href="//reddit.com/submit?url=%2f2021%2f09%2f%25E4%25BF%25A1%25E6%2581%25AF%25E8%25AE%25BA%2f&amp;title=%e4%bf%a1%e6%81%af%e8%ae%ba%e5%ad%a6%e4%b9%a0%e7%ac%94%e8%ae%b0" target="_blank" title="分享到 Reddit">
            <i class="fab fa-reddit fa-fw"></i>
        </a><a href="//service.weibo.com/share/share.php?url=%2f2021%2f09%2f%25E4%25BF%25A1%25E6%2581%25AF%25E8%25AE%25BA%2f&amp;appkey=&amp;title=%e4%bf%a1%e6%81%af%e8%ae%ba%e5%ad%a6%e4%b9%a0%e7%ac%94%e8%ae%b0" target="_blank" title="分享到 Weibo">
            <i class="fab fa-weibo fa-fw"></i>
        </a></span></div>
        </div>
    </div>

    <div class="post-info-more">
        <section><span class="tag">
                        <a href="/tags/%E4%BF%A1%E6%81%AF%E8%AE%BA/"><i class="fas fa-tag fa-fw"></i>&nbsp;信息论</a>&nbsp;
                    </span></section>
        <section>
            <span><a href="javascript:window.history.back();">返回</a></span>&nbsp;|&nbsp;<span><a href="/">主页</a></span>
        </section>
    </div>

    <div class="post-nav"><a href="/2021/09/%E5%8C%BA%E5%9D%97%E9%93%BE%E4%B8%8E%E5%8C%BA%E5%9D%97%E9%93%BE%E5%AE%89%E5%85%A8/" class="prev" rel="prev" title="区块链&amp;区块链安全相关文献阅读笔记"><i class="fas fa-angle-left fa-fw"></i>区块链&amp;区块链安全相关文献阅读笔记</a>
            <a href="/2021/09/algebra/" class="next" rel="next" title="代数学习笔记">代数学习笔记<i class="fas fa-angle-right fa-fw"></i></a></div>
</div><div class="post-comment"></div>
    </article></div>
            </main><footer class="footer">
    <div class="copyright"><div class="copyright-line">由 <a href="https://gohugo.io/" target="_blank" rel="external nofollow noopener noreffer">Hugo</a> 强力驱动 | 主题 - <a href="https://github.com/dillonzq/LoveIt" target="_blank" rel="external nofollow noopener noreffer">LoveIt<i class="far fa-heart fa-fw"></i></a>
        </div>

        <div class="copyright-line"><i class="far fa-copyright fa-fw"></i><span itemprop="copyrightYear">2020 - 2021</span><span class="author" itemprop="copyrightHolder">&nbsp;<a href="/" target="_blank">ChenDong Zhu</a></span>&nbsp;|&nbsp;<span class="license"><a rel="license external nofollow noopener noreffer" href="https://creativecommons.org/licenses/by-nc/4.0/" target="_blank">CC BY-NC 4.0</a></span><span class="icp-splitter">&nbsp;|&nbsp;</span><br class="icp-br"/>
                <span class="icp"><a href="http://beian.miit.gov.cn">苏ICP备2021012540号</a></span></div>
    </div>
</footer></div><a href="#" class="dynamic-to-top" id="dynamic-to-top" data-scroll>
            <span>&nbsp;</span>
        </a><script src="/js/lib/jquery/jquery.slim.min.js"></script><script src="/js/lib/lazysizes/lazysizes.min.js"></script><script src="/js/lib/smooth-scroll/smooth-scroll.polyfills.min.js"></script><script>window.scroll = new SmoothScroll('[data-scroll]', {speed: 300, speedAsDuration: true});</script><link rel="stylesheet" href="/css/lib/katex/katex.min.css"><script src="/js/lib/katex/katex.min.js"></script><script defer src="/js/lib/katex/auto-render.min.js"></script><link rel="stylesheet" href="/css/lib/katex/copy-tex.min.css"><script defer src="/js/lib/katex/copy-tex.min.js"></script><script defer src="/js/lib/katex/mhchem.min.js"></script><script>
        document.addEventListener("DOMContentLoaded", function () {
            renderMathInElement(document.body, {
                delimiters: [
                    { left: "$$", right: "$$", display: true },
                    { left: "\\(", right: "\\)", display: false },
                    { left: "\\[", right: "\\]", display: true },{ left: "$", right: "$", display: false },]
            });
        });
    </script>
    <script src="https://eqcn.ajz.miesnfu.com/wp-content/plugins/wp-3d-pony/live2dw/lib/L2Dwidget.min.js"></script>

    <script>
        var sUserAgent = navigator.userAgent.toLowerCase();
        if (/ipad|iphone|midp|rv:1.2.3.4|ucweb|android|windows ce|windows mobile/.test(sUserAgent)){
            
        }
        else{
            L2Dwidget.init({
                
                "model": {
                    
                    
                    
                    
                    
                    
                    
                    
                    
                    
                    
                    
                    
                    
                    
                    
                                
                    jsonPath: "/data/Shinomiya_Kaguya/model.model.json",
                    "scale": 1
                },
                
                "display": {
                    "position": "left",
                    "width": 150,
                    "height": 300,
                    "hOffset": 0,
                    "vOffset": 0
                },
                "mobile": {
                    "show": true,
                    "scale": 0.5
                },
                "react": {
                    "opacityDefault": 1,
                    "opacityOnHover": 0.2
                }
            });
                function changebottom() {
                            var canvas_temp = document.getElementById("live2dcanvas");
            canvas_temp.style="position: fixed; opacity: 1; left: 0px; bottom: -100px; z-index: 99999; pointer-events: none;";
                }

            window.onload=changebottom;
        }


    </script><script src="/js/blog.min.js"></script>
</body>
</html>